/** 
 * File-Name:HttpClientAgent.java
 *
 * Created on 2010-11-20 下午08:29:33
 * 
 * @author: Neo (neolimeng@gmail.com)
 * Software Engineering Institute, Peking University, China
 * 
 * Copyright (c) 2009, Peking University
 * 
 *
 */
package org.neocrawler.network;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;

import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.log4j.Logger;
import org.neocrawler.crawler.CrawlerManager;
import org.neocrawler.dao.WebContentDao;
import org.neocrawler.entity.WebContent;
import org.neocrawler.storage.WebPageSaver;

/**
 * Description:
 * 
 * @author: Neo (neolimeng@gmail.com) Software Engineering Institute, Peking
 *          University, China
 * @version 1.0 2010-11-20 下午08:29:33
 */
public class HttpClientAgent {
	private static Logger log = Logger.getLogger(HttpClientAgent.class);
	private static final Integer DEFAULT_CONNECT_TIME_OUT = CrawlerManager
			.getConfiguration().getConnectTimeout();
	private static final Integer DEFAULT_GET_REQUEST_TIME_OUT = CrawlerManager
			.getConfiguration().getGetRequestTimeout();
	private static final boolean DEFAULT_USE_PROXY_OR_NOT = CrawlerManager
			.getConfiguration().isUseProxyOrNot();
	private static final String PROXY_URL = CrawlerManager.getConfiguration()
			.getProxyUrl();
	private static final Integer PROXY_PORT = CrawlerManager.getConfiguration()
			.getProxyPort();
	private static WebContentDao webContentDao = new WebContentDao();

	public static String getMethod(WebContent webContent) {
		return getMethod(webContent, DEFAULT_CONNECT_TIME_OUT,
				DEFAULT_GET_REQUEST_TIME_OUT, false, null);
	}

	public static String getMethod(WebContent webContent, String savePath) {
		return getMethod(webContent, DEFAULT_CONNECT_TIME_OUT,
				DEFAULT_GET_REQUEST_TIME_OUT, true, savePath);
	}

	public static String getMethod(WebContent webContent,
			Integer connectTimeOut, Integer getRequestTimeOut,
			Boolean saveOrNot, String path) {
		if (webContent == null) {
			log.error("getMethod传入的webContent为空！！");
			return null;
		}
		log.info("准备访问：" + webContent.getUuid() + "\t" + webContent.getUrl());
		String pageContent = null;
		GetMethod getMethod = null;
		/* 1 生成 HttpClinet 对象并设置参数 */
		try {
			HttpClient httpClient = new HttpClient();
			if (DEFAULT_USE_PROXY_OR_NOT) {
				httpClient.getHostConfiguration().setProxy(PROXY_URL,
						PROXY_PORT);
			}
			// 设置 Http 连接超时为5秒
			httpClient.getHttpConnectionManager().getParams()
					.setConnectionTimeout(connectTimeOut);
			httpClient.getParams().setCookiePolicy(CookiePolicy.RFC_2109);
			httpClient
					.getParams()
					.setParameter(
							HttpMethodParams.USER_AGENT,
							"Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8");

			/* 2 生成 GetMethod 对象并设置参数 */
			getMethod = new GetMethod(webContent.getUrl().replace(" ", "%20"));
			// 设置 get 请求超时为 5 秒
			getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT,
					getRequestTimeOut);
			// 设置请求重试处理，用的是默认的重试处理：请求三次
			getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
					new DefaultHttpMethodRetryHandler());
			getMethod.getParams().setCookiePolicy(
					CookiePolicy.BROWSER_COMPATIBILITY);

			/* 3 执行 HTTP GET 请求 */
			Integer statusCode = httpClient.executeMethod(getMethod);
			webContent.setHttpStatusCode(statusCode.toString());
			webContent.setHttpStatusLine(getMethod.getStatusLine().toString());
			log.info(getMethod.getStatusLine().toString() + "\t"
					+ webContent.getUrl().replace(" ", "%20"));

			// for (Header h : getMethod.getRequestHeaders()) {
			// System.out.println(h.getName() + " " + h.getValue());
			// }
			/* 4 判断访问的状态码 */
			if (statusCode != HttpStatus.SC_OK) {
				log.info("Method failed: " + getMethod.getStatusLine());
				webContent.setStatus(WebContent.STATUS_ERROR);
				pageContent = null;
			} else {
				/* 5 处理 HTTP 响应内容 */
				// HTTP响应头部信息，这里简单打印
				// Header[] headers = getMethod.getResponseHeaders();
				// for (Header h : headers)
				// System.out.println(h.getName() + " " + h.getValue());
				// 读取 HTTP 响应内容，这里简单打印网页内容
				// ================ 读取小文件 ==============//
				// byte[] responseBody = getMethod.getResponseBody();// 读取为字节数组
				// pageContent = new String(responseBody);
				// System.out.println(new String(pageContent));

				// ================ 读取大文件 ==============//
				// 读取为 InputStream，在网页内容数据量大时候推荐使用
				InputStream response = getMethod.getResponseBodyAsStream();
				StringBuffer buffer = new StringBuffer();
				String line; // 用来保存每行读取的内容
				BufferedReader reader = new BufferedReader(
						new InputStreamReader(response));
				line = reader.readLine(); // 读取第一行
				while (line != null) { // 如果 line 为空说明读完了
					buffer.append(line); // 将读到的内容添加到 buffer 中
					buffer.append("\n"); // 添加换行符
					line = reader.readLine(); // 读取下一行
				}
				pageContent = buffer.toString();
				// System.out.println(pageContent);

				// 如果需要保存，这里保存该网页到指定目录
				if (saveOrNot) {
					// String pageFileName =
					// getFileNameByUrl(webContent.getUrl(),
					// getMethod.getResponseHeader("Content-Type")
					// .getValue());
					String pageFileName = webContent.getUuid() + ".html";

					WebPageSaver.saveWebPage(
							path + webContent.getType() + "\\", pageFileName,
							pageContent);

					webContent.setLocalStorePath(path + webContent.getType()
							+ "\\" + pageFileName);
				}
				webContent.setStatus(WebContent.STATUS_OK);
			}
		} catch (Exception e) {
			// 发生网络异常
			webContent.setStatus(WebContent.STATUS_EXCEPTION);
			webContent.setExceptionName(e.getClass().getName());
			String exceptionStackTraceString = "";
			exceptionStackTraceString += e.getClass().getName() + ": "
					+ e.getMessage() + "\n";
			for (int j = 0; e.getStackTrace() != null
					&& j < e.getStackTrace().length; j++) {
				exceptionStackTraceString += "\t at "
						+ e.getStackTrace()[j].toString() + "\n";
			}
			webContent.setExceptionStackTrace(exceptionStackTraceString);
			e.printStackTrace();
		} finally {
			/* 6 .释放连接 */
			if (getMethod != null) {
				getMethod.releaseConnection();
			}
			if (saveOrNot) {
				webContentDao.update(webContent);
			}
		}
		return pageContent;
	}

	/**
	 * 根据 url 和网页类型生成需要保存的网页的文件名 去除掉 url 中非文件名字符
	 */
	public static String getFileNameByUrl(String url, String contentType) {
		url = new String(url.substring(7));// remove http://
		if (contentType.indexOf("html") != -1)// text/html
		{
			url = url.replaceAll("[ ;\\?/:*|<>\"]", "_#_") + ".html";
			return url;
		} else// 如application/pdf
		{
			return url.replaceAll("[ ;\\?/:*|<>\"]", "_#_")
					+ "."
					+ new String(contentType.substring(contentType
							.lastIndexOf("/") + 1));
		}
	}

	public static void main(String[] args) {
		WebContent webContent = new WebContent();
		webContent.setUrl("http://www.pku.edu.cn");
		HttpClientAgent.getMethod(webContent);

		// String fileName =
		// "www.v-grid.info_#_wsdl_#_escience_#_CallForPapers.wsdl.xml; charset=\"utf-8\"";
		// System.out.println(HttpClientAgent.getFileNameByUrl(fileName,
		// "html"));
	}
}
